{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RIuaiSsdHnuA"
      },
      "source": [
        "# A colab for \"What Matters In On-Policy Reinforcement Learning? A Large-Scale Empirical Study\" paper \n",
        "https://arxiv.org/abs/2006.05990\n",
        "\n",
        "\n",
        "Experiment results are stored in a pickled file available in Google Cloud Storage bucket.\n",
        "This colab reads the file and generates the plots."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wbni7sSXGqQb"
      },
      "outputs": [],
      "source": [
        "#@title Authentication to access a datadump from Google Cloud Storage bucket.\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eMFt8OaAG6ST",
        "outputId": "950fc64b-ec00-4365-91cb-aee48c182a35"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Copying gs://seed_rl_external_data_release/mujoco123_data.dump...\n",
            "\\ [1 files][370.2 MiB/370.2 MiB]                                                \n",
            "Operation completed over 1 objects/370.2 MiB.                                    \n"
          ]
        }
      ],
      "source": [
        "#@title Copy the data\n",
        "!gsutil cp gs://seed_rl_external_data_release/mujoco123_data.json ."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ij4o2T40HGXz"
      },
      "outputs": [],
      "source": [
        "#@title Load the data\n",
        "import json\n",
        "import pandas as pd\n",
        "\n",
        "with open('mujoco123_data.json' , 'r') as f:\n",
        "  tmp = json.load(f)\n",
        "\n",
        "loaded_data = {}\n",
        "\n",
        "def convert_data(x):\n",
        "  x = dict([(int(k), v) for k, v in x.items()])\n",
        "  return pd.Series(x)\n",
        "\n",
        "for k, v in tmp.items():\n",
        "  loaded_data[int(k)] = v\n",
        "  df = pd.read_json(v[0])\n",
        "  print(f'starting to process {k} {len(df)}')\n",
        "  df['_eval_'] = df['_eval_'].map(convert_data)\n",
        "  df['_eval_original_avg'] = df['_eval_original_avg'].map(convert_data)  \n",
        "  loaded_data[int(k)][0] = df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "d6sbZhJcIZ3x"
      },
      "outputs": [],
      "source": [
        "#@title Utils\n",
        "\n",
        "#@title Overwrite dict\n",
        "\n",
        "overwrite_dict = {\n",
        "    #Experiment names\n",
        "    'final_losses': '\\\\texttt{Policy Losses}',\n",
        "    'final_time': '\\\\texttt{Time}',\n",
        "    'final_optimize': '\\\\texttt{Optimizers}',\n",
        "    'final_arch': '\\\\texttt{Networks architecture}',\n",
        "    'final_arch2': '\\\\texttt{Networks architecture}',\n",
        "    'final_stability': '\\\\texttt{Normalization and clipping}',\n",
        "    'final_advantages': '\\\\texttt{Advantage Estimation}',\n",
        "    'final_setup': '\\\\texttt{Training setup}',\n",
        "    'final_regularizer': '\\\\texttt{Regularizers}',\n",
        "    # Batch mode things.\n",
        "    '_gin.study_design.choice_value_batch_mode': '\\\\choicet{batchhandling}',\n",
        "    'repeat': 'Fixed trajectories',\n",
        "    'shuffle': 'Shuffle trajectories',\n",
        "    'split': 'Shuffle transitions',\n",
        "    'split_with_advantage_recomputation': 'Shuffle transitions (recompute advantages)',\n",
        "\n",
        "    '_gin.study_design.choice_value_epochs_per_step': '\\\\choicet{numepochsperstep}',\n",
        "    '_gin.study_design.choice_value_num_actors_in_learner': '\\\\choicet{numenvs}',\n",
        "    '_gin.study_design.choice_value_batch_size_transitions': '\\\\choicet{batchsize}',\n",
        "    '_gin.study_design.choice_value_step_size_transitions': '\\\\choicet{stepsize}',\n",
        "\n",
        "    '_gin.study_design.choice_value_value_loss': '\\\\choicet{valueloss}',\n",
        "\n",
        "    '_gin.study_design.choice_value_ppo_style_value_clipping_epsilon': '\\\\choicet{ppovalueclip}',\n",
        "\n",
        "    '_gin.study_design.choice_value_sub_advantage_estimator_gae_gae_lambda': '\\\\choicet{gaelambda}',\n",
        "    '_gin.study_design.choice_value_advantage_estimator': '\\\\choicet{advantageestimator}',\n",
        "    '_gin.study_design.choice_value_sub_advantage_estimator_n_step_n': '\\\\choicet{nstep}',\n",
        "    '_gin.study_design.choice_value_sub_value_loss_huber_delta': '\\\\choicet{huberdelta}',\n",
        "    '_gin.study_design.choice_value_sub_advantage_estimator_v_trace_lambda': '\\\\choicet{vtraceaelambda}',\n",
        "    '_gin.study_design.choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation': '\\\\choicet{vtraceaecrho}',\n",
        "\n",
        "\n",
        "    '_gin.study_design.choice_value_discount_factor': '\\\\choicet{discount}',\n",
        "\n",
        "    '_gin.study_design.choice_value_frame_skip': '\\\\choicet{frameskip}',\n",
        "\n",
        "    '_gin.study_design.choice_value_handle_abandoned_episodes_properly': '\\\\choicet{handleabandon}',\n",
        "\n",
        "    '_gin.study_design.choice_value_learning_rate': '\\\\choicet{adamlr}',\n",
        "    # Losses.\n",
        "\n",
        "    '_gin.study_design.choice_value_standard_policy_losses': '\\\\choicet{policyloss}',\n",
        "    'repeat_positive_advantages':'RPA',\n",
        "    '_gin.study_design.choice_value_sub_standard_policy_losses_v_trace_vtrace_max_importance_weight':'\\\\choicet{vtracelossrho}',\n",
        "    '_gin.study_design.choice_value_sub_standard_policy_losses_ppo_ppo_epsilon':'\\\\choicet{ppoepsilon}',\n",
        "    '_gin.study_design.choice_value_sub_standard_policy_losses_v_mpo_vmpo_e_n':'\\\\choicet{vmpoeps}',\n",
        "    '_gin.study_design.choice_value_sub_standard_policy_losses_awr_awr_beta':'\\\\choicet{awrbeta}',\n",
        "    '_gin.study_design.choice_value_sub_standard_policy_losses_awr_awr_w_max':'\\\\choicet{awrw}',\n",
        "\n",
        "    # Optimizer.\n",
        "    '_gin.study_design.choice_value_learning_rate_decay': '\\\\choicet{lrdecay}',\n",
        "    '_gin.study_design.choice_value_optimizer':'\\\\choicet{optimizer}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_adam_momentum':'\\\\choicet{adammom}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_adam_epsilon':'\\\\choicet{adameps}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_adam_learning_rate':'\\\\choicet{adamlr}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_rmsprop_centered':'\\\\choicet{rmscent}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_rmsprop_momentum': '\\\\choicet{rmsmom}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_rmsprop_epsilon': '\\\\choicet{rmseps}',\n",
        "    '_gin.study_design.choice_value_sub_optimizer_rmsprop_learning_rate': '\\\\choicet{rmslr}',\n",
        "\n",
        "    # Architecture.\n",
        "    '_gin.study_design.choice_value_action_postprocessing': '\\\\choicet{actionpost}',\n",
        "    '_gin.study_design.choice_value_last_kernel_init_value_scaling': '\\\\choicet{valueinit}',\n",
        "    '_gin.study_design.choice_value_std_independent_of_input': '\\\\choicet{stdind}',\n",
        "    '_gin.study_design.choice_value_last_kernel_init_policy_scaling': '\\\\choicet{policyinit}',\n",
        "    '_gin.study_design.choice_value_scale_function': '\\\\choicet{stdtransform}',\n",
        "    '@safe_exp': 'exp',\n",
        "    '@tf.nn.softplus': 'softplus',\n",
        "    '_gin.study_design.choice_value_initial_action_std': '\\\\choicet{initialstd}',\n",
        "    '_gin.study_design.choice_value_initializer': '\\\\choicet{init}',\n",
        "    '@GlorotNormal()': 'Glorot normal',\n",
        "    '@GlorotUniform()': 'Glorot uniform',\n",
        "    '@Orthogonal()': 'Orthogonal',\n",
        "    '@he_normal()': 'He normal',\n",
        "    '@he_uniform()': 'He uniform',\n",
        "    '@lecun_normal()': 'LeCun normal',\n",
        "    '@lecun_uniform()': 'LeCun uniform',\n",
        "    '@orthogonal_gain_sqrt2()': 'Orthogonal(gain=1.41)',\n",
        "    '_gin.study_design.choice_value_policy_and_value_function_network': '\\\\choicet{mlpshared}',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_shared_policy_mlp_width': '\\\\choicet{sharedwidth}',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_shared_policy_mlp_depth': '\\\\choicet{shareddepth}',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_shared_baseline_cost': '\\\\choicet{baselinecost}',\n",
        "    '_gin.study_design.choice_value_minimum_action_std': '\\\\choicet{minstd}',\n",
        "    '_gin.study_design.choice_value_activation': '\\\\choicet{activation}',\n",
        "    '@swish': 'Swish',\n",
        "    '@tf.nn.elu': 'ELU',\n",
        "    '@tf.nn.leaky_relu': 'Leaky ReLU',\n",
        "    '@tf.nn.relu': 'ReLU',\n",
        "    '@tf.nn.sigmoid': 'Sigmoid',\n",
        "    '@tf.nn.tanh': 'Tanh',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_separate_policy_mlp_width': '\\\\choicet{policywidth}',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_separate_policy_mlp_depth':'\\\\choicet{policydepth}',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_separate_value_mlp_width':'\\\\choicet{valuewidth}',\n",
        "    '_gin.study_design.choice_value_sub_policy_and_value_function_network_separate_value_mlp_depth':'\\\\choicet{valuedepth}',\n",
        "    '_gin.study_design.choice_value_policy_mlp_width':'\\\\choicet{policywidth}',\n",
        "    '_gin.study_design.choice_value_value_mlp_depth':'\\\\choicet{valuedepth}',\n",
        "    '_gin.study_design.choice_value_value_mlp_width':'\\\\choicet{valuewidth}',\n",
        "    '_gin.study_design.choice_value_policy_mlp_depth':'\\\\choicet{policydepth}',\n",
        "\n",
        "    '_gin.study_design.choice_value_ppo_epsilon': '\\\\choicet{ppoepsilon}',\n",
        "    '_gin.study_design.choice_value_input_normalization': '\\\\choicet{norminput}',\n",
        "    'Avg (comp=False)': 'Average',\n",
        "    '_gin.study_design.choice_value_gradient_clipping': '\\\\choicet{clipgrad}',\n",
        "    '_gin.study_design.choice_value_normalize_advantages': '\\\\choicet{normadv}',\n",
        "    '_gin.study_design.choice_value_reward_normalization': '\\\\choicet{normreward}',\n",
        "    'Popart Avg (comp=False)': 'Average',\n",
        "    '_gin.study_design.choice_value_sub_input_normalization_avg_compfalse_input_clipping': '\\\\choicet{clipinput}',\n",
        "\n",
        "    # Regularization.\n",
        "    '_gin.study_design.choice_value_policy_regularization': '\\\\choicet{regularizationtype}',\n",
        "    '_gin.study_design.choice_value_sub_policy_regularization_constraint_regularization_constraint': '\\\\choicet{regularizerconstraint}',\n",
        "    '_gin.study_design.choice_value_sub_policy_regularization_penalty_regularization_penalty': '\\\\choicet{regularizerpenalty}',\n",
        "    \n",
        "    '_gin.study_design.choice_value_sub_regularization_constraint_klmupi_kl_mu_pi_threshold': '\\\\choicet{regularizerconstraintklmupi}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_constraint_klpimu_kl_pi_mu_threshold': '\\\\choicet{regularizerconstraintklpimu}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_constraint_klrefpi_kl_ref_pi_threshold': '\\\\choicet{regularizerconstraintklrefpi}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_constraint_decoupled_klmupi_kl_mu_pi_mean_threshold': '\\\\choicet{regularizerconstraintklmupimean}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_constraint_decoupled_klmupi_kl_mu_pi_std_threshold': '\\\\choicet{regularizerconstraintklmupistd}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_constraint_entropy_entropy_threshold': '\\\\choicet{regularizerconstraintentropy}',\n",
        "\n",
        "    '_gin.study_design.choice_value_sub_regularization_penalty_klmupi_coefficient': '\\\\choicet{regularizerpenaltyklmupi}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_penalty_klpimu_coefficient': '\\\\choicet{regularizerpenaltyklpimu}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_penalty_klrefpi_coefficient': '\\\\choicet{regularizerpenaltyklrefpi}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_penalty_decoupled_klmupi_mean_coefficient': '\\\\choicet{regularizerpenaltyklmupimean}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_penalty_decoupled_klmupi_std_coefficient': '\\\\choicet{regularizerpenaltyklmupistd}',\n",
        "    '_gin.study_design.choice_value_sub_regularization_penalty_entropy_coefficient': '\\\\choicet{regularizerpenaltyentropy}',\n",
        "\n",
        "    'constraint': 'Constraint',\n",
        "    'penalty': 'Penalty',\n",
        "    'no regularization': 'No regularization',\n",
        "  }\n",
        "\n",
        "\n",
        "\n",
        "import sys\n",
        "if sys.version_info[0] \u003c 3:\n",
        "  raise Exception(\"Must be using Python 3\")\n",
        "\n",
        "\n",
        "from concurrent import futures\n",
        "import hashlib\n",
        "import os\n",
        "import copy\n",
        "import time\n",
        "import dill\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import math\n",
        "import six.moves.cPickle\n",
        "\n",
        "\n",
        "start_time = time.time()\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "from matplotlib import lines\n",
        "\n",
        "import copy\n",
        "from io import BytesIO\n",
        "\n",
        "from scipy.stats import binom\n",
        "\n",
        "import base64\n",
        "\n",
        "# These are the \"Tableau 20\" colors as RGB.\n",
        "tableau20 = [(31, 119, 180), (174, 199, 232), (255, 127, 14), (255, 187, 120),\n",
        "             (44, 160, 44), (152, 223, 138), (214, 39, 40), (255, 152, 150),\n",
        "             (148, 103, 189), (197, 176, 213), (140, 86, 75), (196, 156, 148),\n",
        "             (227, 119, 194), (247, 182, 210), (127, 127, 127), (199, 199, 199),\n",
        "             (188, 189, 34), (219, 219, 141), (23, 190, 207), (158, 218, 229)]\n",
        "\n",
        "# Scale the RGB values to the [0, 1] range, which is the format matplotlib\n",
        "# accepts.\n",
        "for i in range(len(tableau20)):\n",
        "  r, g, b = tableau20[i]\n",
        "  tableau20[i] = (r / 255., g / 255., b / 255.)\n",
        "\n",
        "\n",
        "\n",
        "def encode_fig_pdf(plt, name):\n",
        "  figfile = BytesIO()\n",
        "  plt.savefig(figfile, format='pdf', bbox_inches=\"tight\", transparent=True)\n",
        "  figfile.seek(0)  # rewind to beginning of file \n",
        "  \n",
        "  return ((name.replace(\".\", \"_\") + '.pdf', figfile.read()))\n",
        "\n",
        "def percentile_ci(vector, p, alpha=0.95):\n",
        "  \"\"\"Computes the pth percentile and a alpha CI based on binomial coverage.\n",
        "  \n",
        "  See https://staff.math.su.se/hoehle/blog/2016/10/23/quantileCI.html. For\n",
        "  efficiency the confidence interval is based on the \n",
        "  [(1-alpha)/2, 1-(1-alpha)/2] interval of a binomial distribution.\n",
        "\n",
        "  Args:\n",
        "    vector: Numpy array with values.\n",
        "    p: Float with percentile (e.g. 0.95).\n",
        "    alpha: Float with coverage of the CI.\n",
        "\n",
        "  Returns:\n",
        "    Percentile as well as a lower and upper bound.\n",
        "  \"\"\"\n",
        "  vector = np.array(vector)\n",
        "  assert vector.ndim == 1\n",
        "  n = vector.shape[0]\n",
        "  low_index, high_index = binom.interval(alpha, n, p, loc=0)\n",
        "  high_index = int(high_index)\n",
        "  f=False\n",
        "  if high_index \u003e n-1:\n",
        "    high_index = n-1\n",
        "    f=True\n",
        "  \n",
        "  percentile = np.percentile(vector, p*100.)\n",
        "  low = vector[int(low_index)]\n",
        "  high = vector[int(high_index)]\n",
        "  return percentile, low, high, f\n",
        "\n",
        "def sanitize(name, replace=True):\n",
        "  if name is None:\n",
        "    return 'None', 0\n",
        "  if len(name) \u003e 0 and name[0] == '\\'':\n",
        "    name = name[1:]\n",
        "  if len(name) \u003e 0 and name[-1] == '\\'':\n",
        "    name = name[:-1]\n",
        "  try:\n",
        "    x = int(name)\n",
        "    return '', (overwrite_dict.get(x, x) if replace else x)\n",
        "  except ValueError:\n",
        "    try:\n",
        "      x = float(name)\n",
        "      return '', (overwrite_dict.get(x, x) if replace else x)\n",
        "    except ValueError:\n",
        "      return (overwrite_dict.get(name, name) if replace else name), 0\n",
        "  return (overwrite_dict.get(name, name) if replace else name), 0\n",
        "\n",
        "\n",
        "def primary_plot95(data,percentile):\n",
        "  fig, axs = plt.subplots(1, 1,sharex=True, figsize=(15, 6),gridspec_kw={'wspace': 0.5})\n",
        "  fig.suptitle('%d%% performance' % percentile)\n",
        "  res=\"\"\n",
        "  groups = data.groupby(['env_name'])['_eval_']\n",
        "  groups_wid = data.groupby(['env_name'])['wid']\n",
        "  w = []\n",
        "\n",
        "  summary = {}\n",
        "  \n",
        "  for (name, group_values), (_, group_wid_values) in zip(groups, groups_wid):\n",
        "    summary[name] = {}\n",
        "    x = [(x.iloc[-1], wid) for x ,wid in zip(group_values.values, group_wid_values.values)]\n",
        "    x.sort()\n",
        "    best_wids = [b for a,b in x][-5:]\n",
        "    x=[a for a,b in x]\n",
        "    summary[name][\"90th percentile\"] = percentile_ci(x, 0.9)[0]\n",
        "    summary[name][\"95th percentile\"] = percentile_ci(x, 0.95)[0]\n",
        "    summary[name][\"99th percentile\"] = percentile_ci(x, 0.99)[0]\n",
        "    summary[name][\"Max\"] = max(x)\n",
        "    summary[name]['best wu'] = '/'.join([str(x) for x in best_wids])\n",
        "    p, l, h, f = percentile_ci(x, percentile / 100.0)\n",
        "    if f:\n",
        "      res += \"\u003cbr/\u003e\u003cb\u003eCAREFUL. Not enough data to compute CI.\u003c/b\u003e\u003cbr/\u003e\"\n",
        "      #print ('CAREFUL. Not enough data to compute CI.')\n",
        "    w.append((sanitize(name), p, p-l, h-p))\n",
        "\n",
        "  summary = pd.DataFrame(summary)\n",
        "\n",
        "  w.sort()\n",
        "  nl = [str(val) if name == '' else name for (name, val), _ , _ , _ in w]\n",
        "  pl = [p for _,  p , _ , _ in w]\n",
        "  ll = [l for _, _ , l , _ in w]\n",
        "  hl = [h for _, _ , _ , h in w]\n",
        "  axs.bar(nl,pl,yerr=[ll, hl], color=tableau20[:len(hl)])#,rotation=90)\n",
        "  axs.set_xticklabels(nl, rotation=90)\n",
        "  axs.set_title('Overall')\n",
        "  \n",
        "  reslatex = \"\"\n",
        "  plt.close()\n",
        "  \n",
        "\n",
        "  res += '\u003cbr/\u003e' + summary.to_html()\n",
        "\n",
        "  summary = summary.drop(\"best wu\")\n",
        "  reslatex += \"\\\\begin{table}[ht]\\n\\\\begin{center}\\n\\\\caption{Performance quantiles across choice configurations.}\\n\\\\label{tab:EXPNAME_overview}\\n\"\n",
        "  reslatex += summary.to_latex(column_format=\"lrrrrr\",float_format=(lambda x: '%.0f' % x)) + '\\n'\n",
        "  reslatex += \"\\\\end{center}\\n\\\\end{table}\"\n",
        "  return res, reslatex\n",
        "\n",
        "def split_data(nl, pl, ll, hl):\n",
        "  nnl = []\n",
        "  npl = []\n",
        "  nll = []\n",
        "  nhl = []\n",
        "  c=0\n",
        "  for n, p, l, h in zip(nl, pl, ll, hl):\n",
        "    if nnl and nnl[-1][0:3] != n[0:3]:\n",
        "      nnl.append(\" \"*c)\n",
        "      c+=1\n",
        "      npl.append(0)\n",
        "      nll.append(0)\n",
        "      nhl.append(0)\n",
        "    nnl.append(n)\n",
        "    npl.append(p)\n",
        "    nll.append(l)\n",
        "    nhl.append(h)\n",
        "  return nnl, npl, nll, nhl\n",
        "\n",
        "def plot95(data, param,percentile,lines=1,height=5):\n",
        "  tenvs = list(set(data['env_name'].values))\n",
        "  envs_list = [tenvs]\n",
        "  if lines == 2:\n",
        "    envs_list = [tenvs[:(len(tenvs)+1)//2], tenvs[(len(tenvs)+1)//2:]]\n",
        "  res = \"\"\n",
        "  pdfs = []\n",
        "  for ei, envs in enumerate(envs_list):\n",
        "    fig, axs = plt.subplots(1, len(envs),sharex=True, figsize=(15 // len(envs_list[0]) * len(envs) * lines , height),gridspec_kw={'wspace': 0.5 / lines})\n",
        "    #fig.suptitle('%dth percentile of performance conditioned on %s \\'%s\\'' % (percentile, 'sub-choice' if '_sub_' in param else 'choice', overwrite_dict.get(param,param)), fontsize=20)\n",
        "    for i, env in enumerate(envs):\n",
        "      edata = data[data['env_name'] == env]\n",
        "      groups = edata.groupby([param])['_eval_']\n",
        "      \n",
        "      w = []\n",
        "      \n",
        "      for name, group_values in groups:\n",
        "        x = [x.iloc[-1] for x in group_values.values]\n",
        "        x.sort()\n",
        "        p, l, h, f = percentile_ci(x, percentile / 100.0)\n",
        "        if f:\n",
        "          res += \"\u003cbr/\u003e\u003cb\u003eCAREFUL. Not enough data to compute CI for %s/%s, len %d.\u003c/b\u003e\u003cbr/\u003e\" % (env,name,len(x))\n",
        "          #print ('CAREFUL. Not enough data to compute CI.')\n",
        "        w.append((sanitize(name), p, p-l, h-p))\n",
        "\n",
        "      w.sort()\n",
        "      nl = [str(val) if name == '' else name for (name, val), _ , _ , _ in w]\n",
        "      pl = [p for _,  p , _ , _ in w]\n",
        "      ll = [l for _, _ , l , _ in w]\n",
        "      hl = [h for _, _ , _ , h in w]\n",
        "      # Hack\n",
        "      if param == '_gin.study_design.choice_value_sub_regularization_constraint_entropy_entropy_threshold':\n",
        "        nl = [\"-\"+x if x!=\"0.0\" else x for x in nl]\n",
        "      if 'custom' in param:\n",
        "        nl, pl, ll, hl = split_data(nl, pl, ll, hl)\n",
        "      axs[i].bar(nl,pl,yerr=[ll, hl], color=tableau20[:len(hl)])#,rotation=90)\n",
        "      axs[i].set_xticklabels(nl, rotation=90)\n",
        "      axs[i].set_title(env, fontsize=20)\n",
        "      \n",
        "\n",
        "    suffix = ''\n",
        "    if lines \u003e 1:\n",
        "      suffix = '_' + str(ei)\n",
        "    if height != 5:\n",
        "      suffix += '_height_'+str(height)\n",
        "    pdfs.append(encode_fig_pdf(plt, 'perf_' + param + suffix))\n",
        "\n",
        "    #reslatex = pdf_image(\"perf_\" + param.replace(\".\", \"_\") + \".pdf\")\n",
        "    plt.close()#show()\n",
        "  \n",
        "  return res, pdfs\n",
        "  \n",
        "\n",
        "def plot_top_90(data, param,percentile):\n",
        "  if 'custom' in param:\n",
        "    return \"\", \"\"\n",
        "  envs = set(data['env_name'].values)\n",
        "  pv = set(data[param].values)\n",
        "  if None in pv:\n",
        "    pv.remove(None)\n",
        "  inv_env = {}\n",
        "  for i, e in enumerate(envs):\n",
        "    inv_env[e] = i\n",
        "  inv_env[\"all\"] = len(envs)\n",
        "  envs.add(\"all\")\n",
        "  fig, axs = plt.subplots(1, len(envs),sharex=True, figsize=(15, 5),gridspec_kw={'wspace': 0.5})\n",
        "  #fig.suptitle('Distribution of %s \\'%s\\' for top %d%% configurations' % ('sub-choice' if '_sub_' in param else 'choice', overwrite_dict.get(param, param), 100-percentile), fontsize=20)\n",
        "  res=\"\"\n",
        "\n",
        "  tdata = data[data[param].notnull()]\n",
        "  groups = tdata.groupby(['env_name'])\n",
        "  data = {name: (group.name,\n",
        "                  [x for x in group.values],\n",
        "                  [x for x in group_v.values],\n",
        "                    )\n",
        "    for (name, group), (_, group_v) in zip(groups[param],\n",
        "                                           groups['_eval_'])}\n",
        "  allv=[]\n",
        "  for a, (b, c, d) in data.items():\n",
        "    p90 = [(x.iloc[-1], i) for i, x in enumerate(d)]\n",
        "    p90 = sorted(p90)\n",
        "    p90 = p90[-((len(p90) - 1) // (100 // (100 - percentile)) + 1):]\n",
        "    v=[data[a][1][x] for _, x in p90]\n",
        "    allv.extend(v)\n",
        "    data[a] = (data[a][0], v)\n",
        "  data[\"all\"] = (\"all\", allv)\n",
        "                                           \n",
        "  s = {}\n",
        "  tt = {}\n",
        "  for (a), (_, c) in data.items():\n",
        "    if a not in tt:\n",
        "      tt[a]=0\n",
        "    if a not in s: \n",
        "      s[a] = {}\n",
        "      for ww in pv:\n",
        "        s[a][ww] = 0\n",
        "    for x in c:\n",
        "      if x not in pv:\n",
        "        continue\n",
        "      tt[a] = tt.get(a,0) + 1\n",
        "      s[a][x] = s[a].get(x, 0) + 1\n",
        "  for i, (a, b) in enumerate(s.items()):\n",
        "    w = []\n",
        "    for x, y in b.items():\n",
        "      w.append((sanitize(x), y/(tt[a]+1e-6),0,0))\n",
        "    \n",
        "    w.sort()\n",
        "    nl = [str(val) if name == '' else name for (name, val), _ , _ , _ in w]\n",
        "    pl = [p for _,  p , _ , _ in w]\n",
        "    ll = [l for _, _ , l , _ in w]\n",
        "    hl = [h for _, _ , _ , h in w]\n",
        "    # Hack\n",
        "    if param == '_gin.study_design.choice_value_sub_regularization_constraint_entropy_entropy_threshold':\n",
        "      nl = [\"-\"+x if x!=\"0.0\" else x for x in nl]        \n",
        "    axs[ inv_env[a] ].bar(nl,pl,yerr=[ll, hl], color=tableau20[:len(hl)])#,rotation=90)\n",
        "    axs[ inv_env[a] ].set_xticklabels(nl, rotation=90)\n",
        "    axs[ inv_env[a] ].set_title(a, fontsize=20)\n",
        "\n",
        "\n",
        "\n",
        "  pdf = encode_fig_pdf(plt, 'frequency_' + param)\n",
        "  plt.close()#show()\n",
        "  \n",
        "  return res, pdf\n",
        "\n",
        "step_limit_easy = 1000000\n",
        "step_limit_hard = 2000000\n",
        "use_average = True\n",
        "\n",
        "percentile = 95\n",
        "\n",
        "\n",
        "RANDOM_POLICY = {\n",
        "    'HalfCheetah-v1': -290,\n",
        "    'Hopper-v1': 18,\n",
        "    'Walker2d-v1': 2,\n",
        "    'Ant-v1': 50,\n",
        "    'Humanoid-v1': 125,\n",
        "}\n",
        "easy_envs = set(['HalfCheetah-v1', 'Hopper-v1', 'Walker2d-v1'])\n",
        "\n",
        "\n",
        "def plot95correlation(data, param1, param2, ename, percentile):\n",
        "  t = time.time()\n",
        "  envs = set(data['env_name'].values)\n",
        "  p1 = [sanitize(x) for x in list(set(data[param1].values)) if x is not None]\n",
        "  p1.sort()\n",
        "  fig, axs = plt.subplots(len(p1), len(envs),sharex=True, figsize=(15, 4 * len(p1)),gridspec_kw={'wspace': 0.5})\n",
        "  #fig.suptitle('%d%% performance' % percentile)\n",
        "  res=\"\"\n",
        "  #print ('Estimated 1 ', time.time() - t )\n",
        "  for i, env in enumerate(envs):\n",
        "    #print ('EstimatedA  ',i, env, time.time() - t )\n",
        "\n",
        "    edata = data[data['env_name'] == env]\n",
        "    groups = edata.groupby([param1, param2])['_eval_']\n",
        "    \n",
        "    for ii, t in enumerate(p1):\n",
        "      w = []\n",
        "      for name, group_values in groups:\n",
        "        if sanitize(name[0]) != t:\n",
        "          continue\n",
        "        x = [x.iloc[-1] for x in group_values.values]\n",
        "        x.sort()\n",
        "        p, l, h, f = percentile_ci(x, percentile / 100.0)\n",
        "        if f:\n",
        "          res += \"\u003cbr/\u003e\u003cb\u003eCAREFUL. Not enough data to compute CI.\u003c/b\u003e\u003cbr/\u003e\"\n",
        "          #print ('CAREFUL. Not enough data to compute CI.')\n",
        "        #print (env, name, x)\n",
        "        w.append((sanitize(name[1]), p, p-l, h-p))\n",
        "\n",
        "      w.sort()\n",
        "      nl = [str(val) if name == '' else name for (name, val), _ , _ , _ in w]\n",
        "      pl = [p for _,  p , _ , _ in w]\n",
        "      ll = [l for _, _ , l , _ in w]\n",
        "      hl = [h for _, _ , _ , h in w]\n",
        "      axs[ii, i].bar(nl,pl,yerr=[ll, hl], color=tableau20[:len(hl)])#,rotation=90)\n",
        "      axs[ii, i].set_xticklabels(nl, rotation=90)\n",
        "      axs[ii, i].set_title(env)\n",
        "      if i == 0:\n",
        "        axs[ii, i].set(ylabel=(str(t[1]) if t[0] == '' else t[0]))\n",
        "  \n",
        "  pdf = encode_fig_pdf(plt, 'correlation_%s_%s_%s' % (ename, param1.replace(\".\", \"_\").replace(\"_gin_study_design_choice\", \"\"), param2.replace(\".\", \"_\").replace(\"_gin_study_design_choice\", \"\")))\n",
        "\n",
        "  plt.close()\n",
        "  \n",
        "  return res, pdf\n",
        "\n",
        "def add_line(str, level=0):\n",
        "  return (\" \" * level) + str + \"\\n\"\n",
        "\n",
        "def add_sub_params(param,paramshort, params, frame, vals, level):\n",
        "  #print ('analyze', param)\n",
        "  latex = \"\"\n",
        "  sparam = paramshort[len('_gin.study_design.choice_value_'):] \n",
        "  vfirst = True\n",
        "  pp = []\n",
        "  #print (param, vals)\n",
        "  for val in vals:\n",
        "    first = True\n",
        "    tsparam = sparam + '_' + str(val).lower().replace('-', '_').replace(' ','_').replace('(','').replace(')','').replace('=','').replace('|', '')\n",
        "    #print (tsparam)\n",
        "    for p in params:\n",
        "      if p.startswith(\"_gin.study_design.choice_value_sub_\" + tsparam):\n",
        "        pp.append(p)\n",
        "        if first:\n",
        "          if vfirst:\n",
        "            vfirst = False\n",
        "            latex += add_line(\"\\\\begin{itemize}\", level-4)\n",
        "          tv = sanitize(val)\n",
        "          ttv0 = tv[0]\n",
        "          ttv0 = ttv0.replace(\"KL(mu||pi)\", \"$\\\\kl(\\\\mu||\\\\pi)$\").replace(\"KL(pi||mu)\", \"$\\\\kl(\\\\pi||\\\\mu)$\").replace(\"KL(ref||pi)\", \"$\\\\kl(\\\\texttt{ref}||\\\\pi)$\")\n",
        "          tv = ttv0, tv[1]\n",
        "          latex += add_line(\"\\\\item For the case ``%s = %s'', we further sampled the sub-choices:\" % (overwrite_dict.get(param, param).replace(\"_\", \"\\\\_\"), str(tv[1]) if tv[0]=='' else tv[0]), level)\n",
        "          latex += add_line(\"\\\\begin{itemize}\", level)\n",
        "        first = False\n",
        "\n",
        "        vz = []\n",
        "        vzorig = []\n",
        "        t = []\n",
        "        torig = []\n",
        "        for x in frame[p].value_counts().keys():\n",
        "          t.append(sanitize(x))\n",
        "        for x in frame[p].value_counts().keys():\n",
        "          torig.append(sanitize(x, replace=False))\n",
        "        t.sort()\n",
        "        torig.sort()\n",
        "        for a, b in t:\n",
        "          if a == '':\n",
        "            # Hack\n",
        "            if p == '_gin.study_design.choice_value_sub_regularization_constraint_entropy_entropy_threshold':\n",
        "              if b \u003e 0:\n",
        "                vz.append(str(-b))\n",
        "              else:\n",
        "                vz.append(str(b))\n",
        "            else:\n",
        "              vz.append(str(b))\n",
        "          else:\n",
        "            if p == '_gin.study_design.choice_value_batch_mode':\n",
        "              vz.append('\\\\texttt{'+a.replace(\"_\", \"\\\\_\")+'}')\n",
        "            else:\n",
        "              vz.append(a.replace(\"_\", \"\\\\_\"))\n",
        "            vz[-1] = vz[-1].replace(\"KL(mu||pi)\", \"$\\\\kl(\\\\mu||\\\\pi)$\").replace(\"KL(pi||mu)\", \"$\\\\kl(\\\\pi||\\\\mu)$\").replace(\"KL(ref||pi)\", \"$\\\\kl(\\\\texttt{ref}||\\\\pi)$\")\n",
        "          \n",
        "        for a, b in torig:\n",
        "          if a == '':\n",
        "            vzorig.append(str(b))\n",
        "          else:\n",
        "            vzorig.append(a)\n",
        "        latex += add_line(\"\\\\item %s: \\\\{%s\\\\}\" % (overwrite_dict.get(p, p).replace(\"_\", \"\\\\_\"), ', '.join(vz)), level+4)\n",
        "        #print (p, 'vs', '_gin.study_design.choice_value'+p[len(\"_gin.study_design.choice_value_sub_\" + tsparam):])\n",
        "        l, tpp = add_sub_params(\n",
        "            p,'_gin.study_design.choice_value'+p[len(\"_gin.study_design.choice_value_sub_\" + tsparam):],\n",
        "            params, frame, vzorig, level=level+8)\n",
        "        latex += l\n",
        "        pp.extend(tpp)\n",
        "    if not first:\n",
        "      latex += add_line(\"\\\\end{itemize}\", level)\n",
        "  if not vfirst:\n",
        "    latex += add_line(\"\\\\end{itemize}\", level-4)\n",
        "  return latex, pp\n",
        "\n",
        "def average(data):\n",
        "  if not data:\n",
        "    return [], []\n",
        "\n",
        "  x = [s for s, _ in data[0].items()]\n",
        "  y = [0] * len(x)\n",
        "\n",
        "  for series in data:\n",
        "    assert len(series) == len(x)\n",
        "    for i, (a, b) in enumerate(series.items()):\n",
        "      assert x[i] == a\n",
        "      y[i] += b / len(data)\n",
        "\n",
        "  return x, y\n",
        "\n",
        "def get_last_score(data):\n",
        "  ind = -1\n",
        "  sc = 0\n",
        "  for a, b in data.items():\n",
        "    if a \u003e ind:\n",
        "      ind = a\n",
        "      sc = b\n",
        "  return b\n",
        "\n",
        "def plot_training_curves(data):\n",
        "  envs = set(data['env_name'].values)\n",
        "  e_inv = {}\n",
        "  for i, env in enumerate(envs):\n",
        "    e_inv[env] = i\n",
        "  fig, axs = plt.subplots(1, len(envs),sharex=False, figsize=(15 * 3, 6),gridspec_kw={'wspace': 0.5 /3})\n",
        "  res=\"\"\n",
        "  for name, group in data.groupby(['env_name']):\n",
        "    ind = e_inv[name]\n",
        "    x = group['_eval_original_avg'].values\n",
        "    y = [(get_last_score(a), i) for i, a in enumerate(x)]\n",
        "    y.sort()\n",
        "    x = [x[b] for a, b in y]\n",
        "    data0 = average(x)\n",
        "    data90 = average(x[-len(x)//10:])\n",
        "    data95 = average(x[-len(x)//20:])\n",
        "    data99 = average(x[-len(x)//100:])\n",
        "\n",
        "    axs[ind].set_title(name, fontsize=20)\n",
        "    axs[ind].plot(data0[0], data0[1], color=tableau20[0], lw=3)\n",
        "    axs[ind].plot(data90[0], data90[1], color=tableau20[1], lw=3)\n",
        "    axs[ind].plot(data95[0], data95[1], color=tableau20[2], lw=3)\n",
        "    axs[ind].plot(data99[0], data99[1], color=tableau20[3], lw=3)\n",
        "    axs[ind].legend(['Overall mean', 'top 10% mean', 'top 5% mean', 'top 1% mean'], fontsize=20)\n",
        "  \n",
        "  pdf = encode_fig_pdf(plt, 'training_curves')\n",
        "  plt.close()\n",
        "  return res, pdf\n",
        "\n",
        "def publish_report(html, pdf_plots, ename, path):\n",
        "  nd = path\n",
        "  d = nd + ename\n",
        "\n",
        "  os.makedirs(d, exist_ok=True)\n",
        "\n",
        "\n",
        "  def save_file(inp):\n",
        "    a, b = inp\n",
        "    path = d + '/' + a\n",
        "    with open(path, 'w' + ('b' if '.pdf' in a else '')) as f:\n",
        "      f.write(b)\n",
        "\n",
        "    print ('%s available at %s' % (a, path))\n",
        "  \n",
        "  with futures.ThreadPoolExecutor(max_workers=100) as executor:\n",
        "    executor.map(save_file, pdf_plots)\n",
        "\n",
        "\n",
        "def generate_config(params, frame):\n",
        "  processed_params = set()\n",
        "  latex = \"\"\n",
        "  latex += add_line(\"\\\\begin{itemize}\")\n",
        "  for param in params:\n",
        "    if not param.startswith('_gin.study_design.choice_value'):\n",
        "      processed_params.add(param)\n",
        "      continue\n",
        "    if param.startswith('_gin.study_design.choice_value') and not param.startswith('_gin.study_design.choice_value_sub_'):\n",
        "      processed_params.add(param)\n",
        "      vals = []\n",
        "      valsorig = []\n",
        "      t = []\n",
        "      torig = []\n",
        "      for x in frame[param].value_counts().keys():\n",
        "        t.append(sanitize(x))\n",
        "        torig.append(sanitize(x, replace=False))\n",
        "      t.sort()\n",
        "      torig.sort()\n",
        "      for a, b in t:\n",
        "        if a == '':\n",
        "          vals.append(str(b))\n",
        "        else:\n",
        "          if param == '_gin.study_design.choice_value_batch_mode':\n",
        "            vals.append('\\\\texttt{'+a.replace(\"_\", \"\\\\_\")+'}')\n",
        "          else:\n",
        "            vals.append(a.replace(\"_\", \"\\\\_\"))\n",
        "      for a, b in torig:\n",
        "        if a == '':\n",
        "          valsorig.append(str(b))\n",
        "        else:\n",
        "          valsorig.append(a)\n",
        "      latex += add_line(\"    \\\\item %s: \\\\{%s\\\\}\" % (overwrite_dict.get(param, param).replace(\"_\", \"\\\\_\").replace(\"#\", \"\\\\#\"), ', '.join(vals)))\n",
        "      l, pp = add_sub_params(param, param, params, frame, valsorig, level=8)\n",
        "      latex += l\n",
        "      for x in pp:\n",
        "        processed_params.add(x)\n",
        "      \n",
        "  latex += add_line(\"\\\\end{itemize}\")\n",
        "  return latex, processed_params\n",
        "\n",
        "def ecdf(data):\n",
        "  \"\"\" Compute ECDF \"\"\"\n",
        "  x = np.sort(data)\n",
        "  n = x.size\n",
        "  y = np.arange(1, n+1) / n\n",
        "  return(x,y)\n",
        "\n",
        "def plot_ecdf_curves(data, param):\n",
        "  envs = set(data['env_name'].values)\n",
        "  e_inv = {}\n",
        "  for i, env in enumerate(envs):\n",
        "    e_inv[env] = i\n",
        "  fig, axs = plt.subplots(1, len(envs),sharex=False, figsize=(15 * 3, 6),gridspec_kw={'wspace': 0.5 /3})\n",
        "  res=\"\"\n",
        "\n",
        "  for e in envs:\n",
        "    sdata = data[data['env_name'] == e]\n",
        "    ind = e_inv[e]\n",
        "    max_val = max([x.iloc[-1] for x in sdata['_eval_']])\n",
        "    min_val = RANDOM_POLICY[e]\n",
        "    leg = []\n",
        "    for name, group in sdata.groupby([param]):\n",
        "      leg.append(sanitize(name)[0])\n",
        "      x = [(x.iloc[-1] - min_val) / (max_val - min_val) for x in group['_eval_'].values]\n",
        "      x, y = ecdf(x)\n",
        "      axs[ind].step(x=x, y=y, where='post')#, color=acolor[name])\n",
        "      axs[ind].set_title('%s (max=%.0f)' % (e, max_val), fontsize=20)\n",
        "    axs[ind].legend(leg, fontsize=20)\n",
        "    \n",
        "  pdf = encode_fig_pdf(plt, 'ecdf_'+param.replace(\".\",\"_\"))\n",
        "  plt.close()\n",
        "  return res, pdf\n",
        "\n",
        "\n",
        "def generate_report(loaded_data, xid, path, BREAK=False):\n",
        "  frame, params, experiment_name, num_seeds, num_wus = loaded_data[xid]\n",
        "\n",
        "  html = \"\"\n",
        "  latex = \"\"\n",
        "  pdf_plots = []\n",
        "\n",
        "\n",
        "  ename = experiment_name.split(' ')[1]\n",
        "  print ('Starting to generate report for %s' % ename)\n",
        "\n",
        "  latex += add_line(\"\\\\clearpage\")\n",
        "  latex += add_line(\"\\\\section{Experiment %s}\" % overwrite_dict.get(ename, ename).replace(\"_\", \"\\\\_\"))\n",
        "  latex += add_line(\"\\\\label{exp_EXPNAME}\")\n",
        "  latex += add_line(\"\\\\subsection{Design}\")\n",
        "  latex += add_line(\"\\\\label{exp_design_EXPNAME}\")\n",
        "  latex += add_line(\"For each of the 5 environments, we sampled %d choice configurations where we sampled the following choices independently and uniformly from the following ranges:\" % (num_wus // 5))\n",
        "  if ename == \"final_arch2\":\n",
        "    tframe, tparams, _, _, _ = loaded_data[13759846] \n",
        "    t, processed_params = generate_config(tparams, tframe)\n",
        "  else:\n",
        "    t, processed_params = generate_config(params, frame)\n",
        "  latex += t\n",
        "\n",
        "  #return\n",
        "  #print(latex)\n",
        "  #return \n",
        "\n",
        "  latex += add_line(\"All the other choices were set to the default values as described in Appendix~\\\\ref{sec:default_settings}.\")\n",
        "  latex += add_line(\"\")\n",
        "  latex += add_line(\"For each of the sampled choice configurations, we train %d agents with different random seeds and compute the performance metric as described in Section~\\\\ref{sec:performance}.\" % num_seeds)\n",
        "\n",
        "  if ename == 'final_arch2':\n",
        "    latex += add_line(\"\")\n",
        "    latex += add_line(\"After running the experiment described above we noticed (Fig.~\\\\ref{fig:final_arch__mlpshared}) that separate policy and value function networks (\\\\choicep{mlpshared}) perform better and we have rerun the experiment with only this variant present.\")\n",
        "    latex += add_line(\"\")\n",
        "    latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "    latex += add_line(\"\\\\begin{center}\")\n",
        "    latex += add_line(\"\\\\centerline{\\\\includegraphics[width=0.45\\\\textwidth]{final_arch/perf__gin_study_design_choice_value_policy_and_value_function_network.pdf}\\\\hspace{1cm}\\\\includegraphics[width=0.45\\\\textwidth]{final_arch/frequency__gin_study_design_choice_value_policy_and_value_function_network.pdf}}\")\n",
        "    latex += add_line(\"\\\\caption{Analysis of choice \\\\choicet{mlpshared}: \"+str(percentile)+ \"th percentile of performance scores conditioned on choice (left) and distribution of choices in top \"+str(100-percentile)+\"\\\\% of configurations (right).}\")\n",
        "    latex += add_line(\"\\\\label{fig:final_arch__mlpshared}\")\n",
        "    latex += add_line(\"\\\\end{center}\")\n",
        "    latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "\n",
        "  latex += add_line(\"\\\\subsection{Results}\")\n",
        "  latex += add_line(\"\\\\label{exp_results_EXPNAME}\")\n",
        "  latex += add_line(\"We report aggregate statistics of the experiment in Table~\\\\ref{tab:EXPNAME_overview} as well as training curves in Figure~\\\\ref{fig:EXPNAME_training_curves}.\")\n",
        "  last = \"fig:EXPNAME_\"+params[-1].replace(\".\",\"_\")\n",
        "  if ename == 'final_setup':\n",
        "    last = \"fig:final_setup2__gin_study_design_choice_value_batch_mode\"\n",
        "  latex += add_line(\"For each of the investigated choices in this experiment, we further provide a per-choice analysis in Figures~\\\\ref{fig:EXPNAME_\"+params[0].replace(\".\",\"_\")+\"}-\\\\ref{\"+last+\"}.\")\n",
        "\n",
        "  html += \"\u003ch1\u003eAnalysis of %s %s \u003c/h1\u003e\" % (xid, experiment_name)\n",
        "\n",
        "  data = frame  \n",
        "  if not BREAK:\n",
        "    t, tl = primary_plot95(data,percentile)\n",
        "    html += t\n",
        "    latex += tl\n",
        "    \n",
        "    t, pdf = plot_training_curves(frame)\n",
        "    html += t\n",
        "    pdf_plots.append(pdf)\n",
        "\n",
        "  \n",
        "  latex += add_line(\"\")\n",
        "  latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "  latex += add_line(\"\\\\begin{center}\")\n",
        "  latex += add_line(\"\\\\centerline{\\\\includegraphics[width=1\\\\textwidth]{EXPNAME/training_curves.pdf}}\")\n",
        "  latex += add_line(\"\\\\caption{Training curves.}\")\n",
        "  latex += add_line(\"\\\\label{fig:EXPNAME_training_curves}\")\n",
        "  latex += add_line(\"\\\\end{center}\")\n",
        "  latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "  if ename == \"final_losses\":\n",
        "    param='_gin.study_design.choice_value_standard_policy_losses'\n",
        "    chtml, pdf = plot_ecdf_curves(frame, param)\n",
        "    pdf_plots.append(pdf)\n",
        "    html += chtml\n",
        "    latex += add_line(\"\")\n",
        "    latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "    latex += add_line(\"\\\\begin{center}\")\n",
        "    latex += add_line(\"\\\\centerline{\\\\includegraphics[width=1\\\\textwidth]{\"+ename+\"/\"+pdf[0]+\"}}\")\n",
        "    latex += add_line(\"\\\\caption{Empirical cumulative density functions of agent performance conditioned on different values of \"+overwrite_dict.get(param,param)+\". The x axis denotes performance rescaled so that 0 corresponds to a random policy and 1 to the best found configuration, and the y axis denotes the quantile.}\")\n",
        "    latex += add_line(\"\\\\label{fig:\"+ename+\"__ecdf_standard_policy_losses}\")\n",
        "    latex += add_line(\"\\\\end{center}\")\n",
        "    latex += add_line(\"\\\\end{figure}\")\n",
        "  \n",
        "  \n",
        "  for param in params:\n",
        "    if BREAK:\n",
        "      break\n",
        "    html += \"\u003ch2\u003eAnalysis of %s\u003c/h2\u003e\" % overwrite_dict.get(param, param)\n",
        "    pname = param.replace(\".\", \"_\") \n",
        "      \n",
        "    if 'custom' in param:\n",
        "      t1, pdfs = plot95(data, param,percentile,lines=2)\n",
        "      html += t1\n",
        "      pdf_plots.extend(pdfs)\n",
        "      \n",
        "      latex += add_line(\"\")\n",
        "      latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "      latex += add_line(\"\\\\begin{center}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=1\\\\textwidth]{EXPNAME/perf_\"+pname+\"_0.pdf}}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=0.65\\\\textwidth]{EXPNAME/perf_\"+pname+\"_1.pdf}}\")\n",
        "      if ename == \"final_losses\":\n",
        "        latex += add_line(\"\\\\caption{Comparison of \"+str(percentile)+ \"th percentile of the performance of different policy losses conditioned on their hyperparameters.}\")\n",
        "      elif ename == \"final_advantages\":\n",
        "        latex += add_line(\"\\\\caption{Comparison of \"+str(percentile)+ \"th percentile of the performance of different advantage estimators conditioned on their hyperparameters.}\")\n",
        "      elif ename == \"final_regularizer\":\n",
        "        latex += add_line(\"\\\\caption{Comparison of \"+str(percentile)+ \"th percentile of the performance of different regularization approaches conditioned on their type.}\")\n",
        "      else:\n",
        "        latex += add_line(\"\\\\caption{Analysis of choice \"+overwrite_dict.get(param,param).replace(\"_\", \"\\\\_\")+\": \"+str(percentile)+ \"th percentile of performance scores conditioned on choice.}\")\n",
        "      latex += add_line(\"\\\\label{fig:EXPNAME_\"+pname+\"}\")\n",
        "      latex += add_line(\"\\\\end{center}\")\n",
        "      latex += add_line(\"\\\\end{figure}\")\n",
        "    else:\n",
        "      if ename == 'final_losses' and param == \"_gin.study_design.choice_value_standard_policy_losses\":\n",
        "        t1, pdfs = plot95(data, param,percentile,height=3)\n",
        "        pdf_plots.extend(pdfs)\n",
        "      if (ename == 'final_arch2' or ename == 'final_arch') and param == \"_gin.study_design.choice_value_initial_action_std\":\n",
        "        t1, pdfs = plot95(data, param,percentile,height=3)\n",
        "        pdf_plots.extend(pdfs)\n",
        "      t1, pdfs = plot95(data, param,percentile)\n",
        "      pdf_plots.extend(pdfs)\n",
        "      html += t1\n",
        "      t2, pdf = plot_top_90(data, param,percentile)\n",
        "      pdf_plots.append(pdf)\n",
        "      html += t2\n",
        "      sub = 'sub-' if '_sub_' in param else ''\n",
        "      latex += add_line(\"\")\n",
        "      latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "      latex += add_line(\"\\\\begin{center}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=0.45\\\\textwidth]{EXPNAME/perf_\"+pname+\".pdf}\\\\hspace{1cm}\\\\includegraphics[width=0.45\\\\textwidth]{EXPNAME/frequency_\"+pname+\".pdf}}\")\n",
        "      latex += add_line(\"\\\\caption{Analysis of choice \"+overwrite_dict.get(param,param).replace(\"_\", \"\\\\_\").replace(\"#\", \"\\\\#\")+\": \"+str(percentile)+ \"th percentile of performance scores conditioned on \"+sub+\"choice (left) and distribution of \"+sub+\"choices in top \"+str(100-percentile)+\"\\\\% of configurations (right).}\")\n",
        "      latex += add_line(\"\\\\label{fig:EXPNAME_\"+pname+\"}\")\n",
        "      latex += add_line(\"\\\\end{center}\")\n",
        "      latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "    if BREAK:\n",
        "      break\n",
        "    #if param.startswith('_gin'):\n",
        "    #  break\n",
        "\n",
        "  if not BREAK:\n",
        "    if ename == \"final_losses\":\n",
        "      param2 = \"_gin.study_design.choice_value_epochs_per_step\"\n",
        "      param1 = \"_gin.study_design.choice_value_standard_policy_losses\"\n",
        "\n",
        "      chtml, pdf = plot95correlation(frame, param1, param2, ename, percentile)\n",
        "      pdf_plots.append(pdf)\n",
        "      html += chtml\n",
        "      latex += add_line(\"\")\n",
        "      latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "      latex += add_line(\"\\\\begin{center}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=1\\\\textwidth]{\"+ename+\"/\"+pdf[0]+\"}}\")\n",
        "      latex += add_line(\"\\\\caption{\"+str(percentile)+\"th percentile of performance scores conditioned on \"+overwrite_dict.get(param1,param1)+\"(rows) and \"+overwrite_dict.get(param2,param2)+\"(bars).}\")\n",
        "      latex += add_line(\"\\\\label{fig:\"+ename+\"__correlation_epochs_per_step_vs_losses}\")\n",
        "      latex += add_line(\"\\\\end{center}\")\n",
        "      latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "\n",
        "    if ename == \"final_optimize\":\n",
        "      param1=\"_gin.study_design.choice_value_sub_optimizer_rmsprop_momentum\"\n",
        "      param2=\"_gin.study_design.choice_value_sub_optimizer_rmsprop_learning_rate\"\n",
        "\n",
        "      chtml, pdf = plot95correlation(frame, param1, param2, ename, percentile)\n",
        "      pdf_plots.append(pdf)\n",
        "      html += chtml\n",
        "      latex += add_line(\"\")\n",
        "      latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "      latex += add_line(\"\\\\begin{center}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=1\\\\textwidth]{\"+ename+\"/\"+pdf[0]+\"}}\")\n",
        "      latex += add_line(\"\\\\caption{\"+str(percentile)+\"th percentile of performance scores conditioned on \\\\choicet{rmsmom}(rows) and \\\\choicet{rmslr}(bars).}\")\n",
        "      latex += add_line(\"\\\\label{fig:\"+ename+\"__correlation_rmsprop_momentum_vs_lr}\")\n",
        "      latex += add_line(\"\\\\end{center}\")\n",
        "      latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "    if False and ename == \"final_setup\":  # we do not want this graph anymore\n",
        "      param1=\"_gin.study_design.choice_value_batch_mode\"\n",
        "      param2=\"_gin.study_design.choice_value_num_actors_in_learner\"\n",
        "\n",
        "      chtml, pdf = plot95correlation(frame, param1, param2, ename, percentile)\n",
        "      pdf_plots.append(pdf)\n",
        "      html += chtml\n",
        "      latex += add_line(\"\")\n",
        "      latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "      latex += add_line(\"\\\\begin{center}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=1\\\\textwidth]{\"+ename+\"/\"+pdf[0]+\"}}\")\n",
        "      latex += add_line(\"\\\\caption{\\todo{Marcin, please change me!!!}}\")\n",
        "      latex += add_line(\"\\\\label{fig:\"+ename+\"__correlation_batch_mode_vs_num_actors}\")\n",
        "      latex += add_line(\"\\\\end{center}\")\n",
        "      latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "    if ename == \"final_setup\":\n",
        "      latex += add_line(\"\")\n",
        "      latex += add_line(\"\\\\begin{figure}[ht]\")\n",
        "      latex += add_line(\"\\\\begin{center}\")\n",
        "      latex += add_line(\"\\\\centerline{\\\\includegraphics[width=0.45\\\\textwidth]{final_setup2/perf__gin_study_design_choice_value_batch_mode.pdf}\\\\hspace{1cm}\\\\includegraphics[width=0.45\\\\textwidth]{final_setup2/frequency__gin_study_design_choice_value_batch_mode.pdf}}\")\n",
        "      latex += add_line(\"\\\\caption{Analysis of choice \\\\choicet{batchhandling}: \"+str(percentile)+\"th percentile of performance scores conditioned on choice (left) and distribution of choices in top \"+str(100-percentile)+\"\\\\% of configurations(right). In order to obtain narrower confidence intervals in this experiment we only sweep \\\\choicet{batchhandling}, \\\\choicet{numenvs}, \\\\choicet{adamlr}.}\")\n",
        "      latex += add_line(\"\\\\label{fig:final_setup2__gin_study_design_choice_value_batch_mode}\")\n",
        "      latex += add_line(\"\\\\end{center}\")\n",
        "      latex += add_line(\"\\\\end{figure}\")\n",
        "\n",
        "  html = html.replace(\"width=\\\"\\\"\", \"width=700\")\n",
        "\n",
        "  latex += add_line(\"\\\\clearpage\")\n",
        "  latex = latex.replace(\"EXPNAMEREADABLE\", \"experiment \" + overwrite_dict.get(ename,ename).replace(\"_\", \"\\\\_\"))\n",
        "  latex = latex.replace(\"EXPNAME\", ename)\n",
        "\n",
        "  pdf_plots.append(('main.tex', latex))\n",
        "\n",
        "  print('Report generation finished for %s' % ename)\n",
        "  publish_report(html, pdf_plots, ename, path)\n",
        "  print ('FINISHED publishing with %d/%s' % (xid, experiment_name))\n",
        "  print('')\n",
        "  print('')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w0sRNsokIws7",
        "outputId": "64f1ec87-5fe8-47b0-c521-e41b0576ab7e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Starting to generate report for final_advantages\n",
            "Report generation finished for final_advantages\n",
            "training_curves.pdf available at final_advantages/training_curves.pdf\n",
            "perf__gin_study_design_choice_value_num_actors_in_learner.pdf available at final_advantages/perf__gin_study_design_choice_value_num_actors_in_learner.pdfperf_custom_advantage_estimator_1.pdf available at final_advantages/perf_custom_advantage_estimator_1.pdf\n",
            "\n",
            "perf_custom_advantage_estimator_0.pdf available at final_advantages/perf_custom_advantage_estimator_0.pdf\n",
            "frequency__gin_study_design_choice_value_value_loss.pdf available at final_advantages/frequency__gin_study_design_choice_value_value_loss.pdf\n",
            "frequency__gin_study_design_choice_value_num_actors_in_learner.pdf available at final_advantages/frequency__gin_study_design_choice_value_num_actors_in_learner.pdf\n",
            "perf__gin_study_design_choice_value_value_loss.pdf available at final_advantages/perf__gin_study_design_choice_value_value_loss.pdf\n",
            "frequency__gin_study_design_choice_value_ppo_style_value_clipping_epsilon.pdf available at final_advantages/frequency__gin_study_design_choice_value_ppo_style_value_clipping_epsilon.pdf\n",
            "perf__gin_study_design_choice_value_ppo_style_value_clipping_epsilon.pdf available at final_advantages/perf__gin_study_design_choice_value_ppo_style_value_clipping_epsilon.pdf\n",
            "perf__gin_study_design_choice_value_advantage_estimator.pdf available at final_advantages/perf__gin_study_design_choice_value_advantage_estimator.pdf\n",
            "frequency__gin_study_design_choice_value_advantage_estimator.pdf available at final_advantages/frequency__gin_study_design_choice_value_advantage_estimator.pdf\n",
            "perf__gin_study_design_choice_value_sub_advantage_estimator_gae_gae_lambda.pdf available at final_advantages/perf__gin_study_design_choice_value_sub_advantage_estimator_gae_gae_lambda.pdf\n",
            "frequency__gin_study_design_choice_value_sub_advantage_estimator_gae_gae_lambda.pdf available at final_advantages/frequency__gin_study_design_choice_value_sub_advantage_estimator_gae_gae_lambda.pdf\n",
            "perf__gin_study_design_choice_value_learning_rate.pdf available at final_advantages/perf__gin_study_design_choice_value_learning_rate.pdf\n",
            "frequency__gin_study_design_choice_value_learning_rate.pdf available at final_advantages/frequency__gin_study_design_choice_value_learning_rate.pdf\n",
            "perf__gin_study_design_choice_value_sub_advantage_estimator_n_step_n.pdf available at final_advantages/perf__gin_study_design_choice_value_sub_advantage_estimator_n_step_n.pdf\n",
            "frequency__gin_study_design_choice_value_sub_advantage_estimator_n_step_n.pdf available at final_advantages/frequency__gin_study_design_choice_value_sub_advantage_estimator_n_step_n.pdf\n",
            "perf__gin_study_design_choice_value_sub_value_loss_huber_delta.pdf available at final_advantages/perf__gin_study_design_choice_value_sub_value_loss_huber_delta.pdf\n",
            "frequency__gin_study_design_choice_value_sub_value_loss_huber_delta.pdf available at final_advantages/frequency__gin_study_design_choice_value_sub_value_loss_huber_delta.pdfperf__gin_study_design_choice_value_sub_advantage_estimator_v_trace_lambda.pdf available at final_advantages/perf__gin_study_design_choice_value_sub_advantage_estimator_v_trace_lambda.pdf\n",
            "\n",
            "frequency__gin_study_design_choice_value_sub_advantage_estimator_v_trace_lambda.pdf available at final_advantages/frequency__gin_study_design_choice_value_sub_advantage_estimator_v_trace_lambda.pdf\n",
            "perf__gin_study_design_choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation.pdf available at final_advantages/perf__gin_study_design_choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation.pdf\n",
            "frequency__gin_study_design_choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation.pdf available at final_advantages/frequency__gin_study_design_choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation.pdf\n",
            "main.tex available at final_advantages/main.tex\n",
            "FINISHED publishing with 13761804/Mujoco123: final_advantages (mujoco)\n",
            "\n",
            "\n",
            "Starting to generate report for final_arch\n",
            "Report generation finished for final_arch\n",
            "frequency__gin_study_design_choice_value_action_postprocessing.pdf available at final_arch/frequency__gin_study_design_choice_value_action_postprocessing.pdf\n",
            "perf__gin_study_design_choice_value_action_postprocessing.pdf available at final_arch/perf__gin_study_design_choice_value_action_postprocessing.pdf\n",
            "training_curves.pdf available at final_arch/training_curves.pdf\n",
            "perf__gin_study_design_choice_value_last_kernel_init_value_scaling.pdf available at final_arch/perf__gin_study_design_choice_value_last_kernel_init_value_scaling.pdf\n",
            "frequency__gin_study_design_choice_value_last_kernel_init_value_scaling.pdf available at final_arch/frequency__gin_study_design_choice_value_last_kernel_init_value_scaling.pdf\n",
            "perf__gin_study_design_choice_value_std_independent_of_input.pdf available at final_arch/perf__gin_study_design_choice_value_std_independent_of_input.pdf\n",
            "frequency__gin_study_design_choice_value_std_independent_of_input.pdf available at final_arch/frequency__gin_study_design_choice_value_std_independent_of_input.pdf\n",
            "perf__gin_study_design_choice_value_last_kernel_init_policy_scaling.pdf available at final_arch/perf__gin_study_design_choice_value_last_kernel_init_policy_scaling.pdf\n",
            "frequency__gin_study_design_choice_value_last_kernel_init_policy_scaling.pdf available at final_arch/frequency__gin_study_design_choice_value_last_kernel_init_policy_scaling.pdf\n",
            "perf__gin_study_design_choice_value_scale_function.pdf available at final_arch/perf__gin_study_design_choice_value_scale_function.pdf\n",
            "frequency__gin_study_design_choice_value_scale_function.pdf available at final_arch/frequency__gin_study_design_choice_value_scale_function.pdf\n",
            "perf__gin_study_design_choice_value_initial_action_std_height_3.pdf available at final_arch/perf__gin_study_design_choice_value_initial_action_std_height_3.pdf\n",
            "perf__gin_study_design_choice_value_initial_action_std.pdf available at final_arch/perf__gin_study_design_choice_value_initial_action_std.pdf\n",
            "frequency__gin_study_design_choice_value_initial_action_std.pdf available at final_arch/frequency__gin_study_design_choice_value_initial_action_std.pdf\n",
            "perf__gin_study_design_choice_value_initializer.pdf available at final_arch/perf__gin_study_design_choice_value_initializer.pdf\n",
            "frequency__gin_study_design_choice_value_initializer.pdf available at final_arch/frequency__gin_study_design_choice_value_initializer.pdf\n",
            "frequency__gin_study_design_choice_value_policy_and_value_function_network.pdf available at final_arch/frequency__gin_study_design_choice_value_policy_and_value_function_network.pdfperf__gin_study_design_choice_value_policy_and_value_function_network.pdf available at final_arch/perf__gin_study_design_choice_value_policy_and_value_function_network.pdf\n",
            "\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_width.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_width.pdf\n",
            "frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_width.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_width.pdf\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_depth.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_depth.pdffrequency__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_depth.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_policy_mlp_depth.pdf\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_baseline_cost.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_baseline_cost.pdf\n",
            "\n",
            "frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_baseline_cost.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_shared_baseline_cost.pdf\n",
            "perf__gin_study_design_choice_value_minimum_action_std.pdf available at final_arch/perf__gin_study_design_choice_value_minimum_action_std.pdf\n",
            "frequency__gin_study_design_choice_value_minimum_action_std.pdf available at final_arch/frequency__gin_study_design_choice_value_minimum_action_std.pdf\n",
            "perf__gin_study_design_choice_value_learning_rate.pdf available at final_arch/perf__gin_study_design_choice_value_learning_rate.pdffrequency__gin_study_design_choice_value_learning_rate.pdf available at final_arch/frequency__gin_study_design_choice_value_learning_rate.pdf\n",
            "perf__gin_study_design_choice_value_activation.pdf available at final_arch/perf__gin_study_design_choice_value_activation.pdf\n",
            "\n",
            "frequency__gin_study_design_choice_value_activation.pdf available at final_arch/frequency__gin_study_design_choice_value_activation.pdf\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_width.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_width.pdf\n",
            "frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_width.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_width.pdf\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_depth.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_depth.pdf\n",
            "frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_depth.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_policy_mlp_depth.pdf\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_width.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_width.pdf\n",
            "frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_width.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_width.pdf\n",
            "perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_depth.pdf available at final_arch/perf__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_depth.pdf\n",
            "frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_depth.pdf available at final_arch/frequency__gin_study_design_choice_value_sub_policy_and_value_function_network_separate_value_mlp_depth.pdf\n",
            "main.tex available at final_arch/main.tex\n",
            "FINISHED publishing with 13759846/Mujoco123: final_arch (mujoco)\n",
            "\n",
            "\n",
            "Starting to generate report for final_arch2\n"
          ]
        }
      ],
      "source": [
        "#@title Generate reports\n",
        "\n",
        "for xid in loaded_data.keys():\n",
        "  generate_report(loaded_data, xid, '', BREAK=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TFq6xLLfMtkC",
        "outputId": "6ce0bad3-32c4-456e-9a6f-5c7a41db966a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "total 388\n",
            "-rw-r--r-- 1 root root 15006 Jan  5 15:43 frequency__gin_study_design_choice_value_advantage_estimator.pdf\n",
            "-rw-r--r-- 1 root root 15261 Jan  5 15:43 frequency__gin_study_design_choice_value_learning_rate.pdf\n",
            "-rw-r--r-- 1 root root 14476 Jan  5 15:43 frequency__gin_study_design_choice_value_num_actors_in_learner.pdf\n",
            "-rw-r--r-- 1 root root 14120 Jan  5 15:43 frequency__gin_study_design_choice_value_ppo_style_value_clipping_epsilon.pdf\n",
            "-rw-r--r-- 1 root root 15252 Jan  5 15:43 frequency__gin_study_design_choice_value_sub_advantage_estimator_gae_gae_lambda.pdf\n",
            "-rw-r--r-- 1 root root 14370 Jan  5 15:43 frequency__gin_study_design_choice_value_sub_advantage_estimator_n_step_n.pdf\n",
            "-rw-r--r-- 1 root root 15022 Jan  5 15:43 frequency__gin_study_design_choice_value_sub_advantage_estimator_v_trace_lambda.pdf\n",
            "-rw-r--r-- 1 root root 13458 Jan  5 15:43 frequency__gin_study_design_choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation.pdf\n",
            "-rw-r--r-- 1 root root 13680 Jan  5 15:43 frequency__gin_study_design_choice_value_sub_value_loss_huber_delta.pdf\n",
            "-rw-r--r-- 1 root root 14769 Jan  5 15:43 frequency__gin_study_design_choice_value_value_loss.pdf\n",
            "-rw-r--r-- 1 root root 10045 Jan  5 15:43 main.tex\n",
            "-rw-r--r-- 1 root root 18163 Jan  5 15:43 perf_custom_advantage_estimator_0.pdf\n",
            "-rw-r--r-- 1 root root 13873 Jan  5 15:43 perf_custom_advantage_estimator_1.pdf\n",
            "-rw-r--r-- 1 root root 15492 Jan  5 15:43 perf__gin_study_design_choice_value_advantage_estimator.pdf\n",
            "-rw-r--r-- 1 root root 15142 Jan  5 15:43 perf__gin_study_design_choice_value_learning_rate.pdf\n",
            "-rw-r--r-- 1 root root 13667 Jan  5 15:43 perf__gin_study_design_choice_value_num_actors_in_learner.pdf\n",
            "-rw-r--r-- 1 root root 15275 Jan  5 15:43 perf__gin_study_design_choice_value_ppo_style_value_clipping_epsilon.pdf\n",
            "-rw-r--r-- 1 root root 14495 Jan  5 15:43 perf__gin_study_design_choice_value_sub_advantage_estimator_gae_gae_lambda.pdf\n",
            "-rw-r--r-- 1 root root 14458 Jan  5 15:43 perf__gin_study_design_choice_value_sub_advantage_estimator_n_step_n.pdf\n",
            "-rw-r--r-- 1 root root 15521 Jan  5 15:43 perf__gin_study_design_choice_value_sub_advantage_estimator_v_trace_lambda.pdf\n",
            "-rw-r--r-- 1 root root 13506 Jan  5 15:43 perf__gin_study_design_choice_value_sub_advantage_estimator_v_trace_max_importance_weight_in_advantage_estimation.pdf\n",
            "-rw-r--r-- 1 root root 14163 Jan  5 15:43 perf__gin_study_design_choice_value_sub_value_loss_huber_delta.pdf\n",
            "-rw-r--r-- 1 root root 14442 Jan  5 15:43 perf__gin_study_design_choice_value_value_loss.pdf\n",
            "-rw-r--r-- 1 root root 18668 Jan  5 15:43 training_curves.pdf\n"
          ]
        }
      ],
      "source": [
        "!ls -l final_advantages"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "what_matters_in_on_policy_rl.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
